package net.tngou.crawler.service;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Set;

import org.apache.commons.collections.map.HashedMap;
import org.apache.commons.collections4.ListUtils;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

import net.tngou.c4j.Crawler;
import net.tngou.c4j.ml.Forecast;
import net.tngou.c4j.ml.Model;
import net.tngou.c4j.model.RegexUrl;
import net.tngou.c4j.model.Url;
import net.tngou.c4j.tag.Mark;
import net.tngou.c4j.util.UrlRegex;
import net.tngou.crawler.dao.DataDao;
import net.tngou.crawler.enums.StateEnum;
import net.tngou.crawler.enums.ValidEnum;
import net.tngou.crawler.jdbc.OrderType;
import net.tngou.crawler.pojo.Crawlerurl;
import net.tngou.crawler.pojo.Data;
import net.tngou.crawler.pojo.Loops;
import net.tngou.crawler.pojo.Ml;
import net.tngou.crawler.pojo.Outvalueurl;
import net.tngou.crawler.pojo.Outvalueurllog;
import net.tngou.crawler.pojo.POJO;
import net.tngou.crawler.pojo.Tag;
import net.tngou.crawler.pojo.Tag_data;
import net.tngou.crawler.pojo.Urldata;
import net.tngou.crawler.pojo.Web;
import net.tngou.crawler.pojo.Weburl;
import net.tngou.crawler.util.DateUtil;
import net.tngou.crawler.util.DigestMD;

/**
 * 
 * @author tngou@tngou.net
 *
 */
public class CrawlerService extends BaseService {
	Crawler crawler = new Crawler();
	
	/**
	 * 设置数据
	 * @param urldata
	 */
	public void saveData(Urldata urldata) {
		Weburl weburl = new Weburl().get(urldata.getWeburl());
		Web web = new Web().get(weburl.getWeb());
		saveUrl(web,urldata.getUrl());
		Data data = crawler.getData(weburl, urldata);
		if(data!=null)
		{
			
			String md16 = DigestMD.MD5To16(data.getValue());
			DataDao dao = new DataDao();
			int total = dao.getTotal(md16, weburl.getId());
			if(total>0||"{}".equals(data.getValue()))
			{
				urldata.setState(StateEnum.无数据.ordinal());
			}else
			{
				data.setMd16(md16);
				long id = data.save();
				if(id>0) {
					saveTag(data);
					saveLoops(data);
				}
				urldata.setState(StateEnum.爬取完成.ordinal());
			}
			
		}else
		{
			urldata.setState(StateEnum.爬取失败.ordinal());
		}
		urldata.update();
	}
	
	public void saveLoops(Data data) {
		JSONObject json = (JSONObject) JSON.parse(data.getValue());
		if(json!=null)
		{
			Set<String> keys = json.keySet();
			keys.forEach(e->{
				Model model = new Model(e, json.getString(e));
				if(model!=null||model.getLoops()!=null||model.getLoops().length>0)
				{
					@SuppressWarnings("unchecked")
					POJO ml = new Ml().get(new HashedMap() {{
						put("keyword", e);
					}});
					String[] loops = model.getLoops();
					for (String string : loops) {
						Loops bean = new Loops();
						bean.setData(data.getId());
						bean.setMd16(string);
						bean.setMl(ml!=null?ml.getId():0);
						bean.save();
					}
				}
			});
		}
	}
	
	public void saveData() {
		 List<Urldata> list = (List<Urldata>) new Urldata().list("state="+StateEnum.新生成.ordinal(), 1, 10, "id", OrderType.ASC);
		 list.forEach(e->{
			 e.setState(StateEnum.爬取中.ordinal());
			 e.update();
		 });
		 list.forEach(e->{
			 saveData(e);
		 });
	}
	
	public void saveUrl(Web web) {
		List<Outvalueurl> outvalueurls = (List<Outvalueurl>) getList("web="+web.getId(), "id", OrderType.ASC, Outvalueurl.class);
		List<Weburl> weburls = (List<Weburl>) getList("web="+web.getId(), "id", OrderType.ASC, Weburl.class);
		List<RegexUrl> outvalueurl_regexs = new ArrayList<>();
		List<RegexUrl> weburl_regexs = new ArrayList<>();
		List<String> baseurls= new ArrayList<>();
		outvalueurls.forEach(e->{
			RegexUrl regex = new RegexUrl();
			regex.setId(e.getWeb());
			regex.setUrl(e.getUrl());
			regex.setType(e.getType());
			outvalueurl_regexs.add(regex);
		});
		weburls.forEach(e->{
			RegexUrl regex = new RegexUrl();
			regex.setId(e.getWeb());
			regex.setUrl(e.getUrl());
			weburl_regexs.add(regex);
		});
		saveUrl(baseurls, web.getUrl(),outvalueurl_regexs, weburl_regexs);
		LocalDateTime t = web.getNext_t().toLocalDateTime().plusMinutes(web.getSpace());
		web.setNext_t(DateUtil.toTimestamp(t));
		web.update();
	}
	
	public void saveUrl(Web web,String url) {
		List<Outvalueurl> outvalueurls = (List<Outvalueurl>) getList("web="+web.getId(), "id", OrderType.ASC, Outvalueurl.class);
		List<Weburl> weburls = (List<Weburl>) getList("web="+web.getId(), "id", OrderType.ASC, Weburl.class);
		List<RegexUrl> outvalueurl_regexs = new ArrayList<>();
		List<RegexUrl> weburl_regexs = new ArrayList<>();
		List<String> baseurls= new ArrayList<>();
		outvalueurls.forEach(e->{
			RegexUrl regex = new RegexUrl();
			regex.setId(e.getWeb());
			regex.setUrl(e.getUrl());
			regex.setType(e.getType());
			outvalueurl_regexs.add(regex);
		});
		weburls.forEach(e->{
			RegexUrl regex = new RegexUrl();
			regex.setId(e.getWeb());
			regex.setUrl(e.getUrl());
			weburl_regexs.add(regex);
		});
		saveUrl(baseurls, url,outvalueurl_regexs, weburl_regexs);
	}
	private void saveUrl(List<String> baseurls,String url,List<RegexUrl> outvalueurls,List<RegexUrl> weburls)
	{
		baseurls.add(url);
		Set<Url> urls = crawler.getUrl(url);		
		UrlRegex regex= new UrlRegex();
		Set<Url> us = regex.filterUrl(urls, weburls);
		us.forEach(e->{
			Urldata bean = new Urldata();
			String md = DigestMD.MD5(e.getUrl());
			if(bean.get("md5='"+md+"'")==null)
			{
				bean.setMd5(md);
				bean.setUrl(e.getUrl());
				bean.setWeburl(e.getId());
				bean.setState(StateEnum.新生成.ordinal());
				bean.save();
			}
		});
		Set<Url> fs = regex.filterUrl(urls, outvalueurls);
		for (Url u : fs) {
			String ul=u.getUrl();
			boolean b=false;
			for (String burl : baseurls) {
				if(ul.equals(burl))b=true;
			}
			if(!b)
			{
				String md = DigestMD.MD5(ul);
				Outvalueurllog bean = new Outvalueurllog();
				if(bean.get("md5='"+md+"'")==null)
				{
					if(u.getType()==0)
					{
						bean.setMd5(md);
						bean.setUrl(ul);
						bean.setOutvalueurl(u.getId());
					}
					saveUrl(baseurls, ul, outvalueurls, weburls);
				}else
				{
					baseurls.add(ul);
				}
				
			}
			
		}
	}
	
	public void saveTag(Data data) {
		List<String> list = Mark.Tags(data);
		Tag bean = new Tag();
		list.forEach(e->{
			@SuppressWarnings("serial")
			Tag tag = bean.get(new HashMap<String ,Object>() {{put("name", e);}});
			if(tag!=null)
			{
				if(tag.getValid()!=ValidEnum.无效.ordinal())
				{
					Tag_data tag_data = new Tag_data();
					tag_data.setData(data.getId());
					tag_data.setTag(tag.getId());
					tag_data.save();
					tag.setCount(tag.getCount()+1);
					tag.update();
					
				}
					
				
			}else
			{
				tag = new Tag();
				tag.setName(e);
				tag.setCount(1);
				long id = tag.save();
				if(id>0)
				{
					Tag_data tag_data = new Tag_data();
					tag_data.setData(data.getId());
					tag_data.setTag(tag.getId());
					tag_data.save();
				}
				
			}
		});
	}
//	
//	public void saveCravlerUrl(Crawlerurl crawlerurl) {
//		if(crawlerurl.getId()>0) {
//			crawlerurl.setState(StateEnum.爬取中.ordinal());
//			crawlerurl.update();
//		}
//		Set<Url> urls = crawler.getUrl(crawlerurl.getUrl());
//		UrlRegex regex= new UrlRegex();
//		Set<Url> us = regex.filterUrl(urls, crawlerurl.getUrl());
//		Onlinepharmacy onlinepharmacy =new Onlinepharmacy().get(crawlerurl.getWeb());
//		JSONArray array=JSON.parseArray(onlinepharmacy.getOurl());
//		List<String> ls = new ArrayList<>();
//		if(array!=null&&!array.isEmpty()) {
//			for (Object string : array) {
//				ls.add(string.toString());
//			}		
//		}
//		if(onlinepharmacy!=null)us=regex.outfilterUrl(us,ls);
//		us.forEach(e->{
//			Crawlerurl bean = new Crawlerurl();
//			String md = DigestMD.MD5(e.getUrl());
//			if(bean.get("md5='"+md+"'")==null)
//			{
//				bean.setMd5(md);
//				bean.setUrl(e.getUrl());
//				bean.setWeb(crawlerurl.getWeb());
//				bean.setState(StateEnum.新生成.ordinal());
//				bean.save();
//			}
//		});
//		if(crawlerurl.getId()>0) {
//		
//			Onlinepharmacy online = new Onlinepharmacy().get(crawlerurl.getWeb());
//			if(online!=null)
//			{
//				array=JSON.parseArray(online.getEurl());
//				ls = new ArrayList<>();
//				if(array!=null&&!array.isEmpty()) {
//					for (Object string : array) {
//						ls.add(string.toString());
//					}		
//				}
//				if(regex.filterUrl( crawlerurl.getUrl(),ls))
//				{
//					crawlerurl.setState(StateEnum.等待上传.ordinal());
//				}else
//				{
//					crawlerurl.setState(StateEnum.无效数据.ordinal());
//				}
//				
//			}else
//			{
//				crawlerurl.setState(StateEnum.爬取完成.ordinal());
//			}			
//			crawlerurl.update();
//		}
//			
//	}
}

